import requests
from bs4 import BeautifulSoup

# 请求
url = 'https://shuihu.5000yan.com/'
# 西游记目录
file = open('data/水浒传.txt', 'w', encoding='utf-8')
# 文本写入对象
response = requests.get(url)
# 请求目录页
soup = BeautifulSoup(response.content)
# 解析目录页
hongloumeng_chapters = soup.find('div', class_='p-3 my-2 bg-white rounded').ul.children
# 获取章回内容

for cpt in hongloumeng_chapters:
    # 迭代访问章节内容
    if cpt != '\n' and cpt is not None:
        link = cpt.a.get('href')
        # 获取章节地址
        cpt_page = requests.get(link)
        # 请求章节页面
        cpt_content=BeautifulSoup(cpt_page.content).find('div',class_='py-2 px-2 px-md-4 bg-white divider')
        # 获取章节内容
        if cpt_content:
            lines = cpt_content.text.splitlines()
            non_empty_lines = [line for line in lines if line.strip()]
            filters_text = '\n'.join(non_empty_lines)
            file.write('\n'+ filters_text + '\n')

file.flush()
# 把缓存中的数据写入文本
file.close()
# 关闭文本流
